# -*- coding: utf-8 -*-
import re

import scrapy


class IndexSpider(scrapy.Spider):
    name = 'index'
    allowed_domains = ['osgeo.cn']
    start_urls = ['https://www.osgeo.cn/scrapy/index.html']
    url = 'https://www.osgeo.cn/scrapy/'

    def parse(self, response):
        url_list = response.xpath('//@href').extract() or response.xpath('//@src').extract()
        for i in url_list:
            # print(i)
            yield scrapy.Request(url=IndexSpider.url + i, callback=self.save_file, cb_kwargs={"name": i})
        yield {
            "name": "index.html",
            "text": response.text,
        }

    def save_file(self, response, name):
        yield {
            "name": name,
            "text": response.text
        }

